"""
    pyspark: 存储数据
    程序没有错，只是环境没有搭好

"""

from __future__ import print_function, division
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession

spark = SparkSession.builder.master("local").appName("test").getOrCreate()
sc = spark.sparkContext

# 读取csv格式
df_csv = spark.read.csv("E:\\Python\\pyspark_demo01\\pyspark_data\\ratings.csv", header=True, )
# 存为csvv格式
df_csv.write.csv("E:\\Python\\pyspark_demo01\\out_dat\\aaa_1.csv", header=True, mode='append')
# 通过coalesce 控制分区数
df_csv.coalesce(1).write.csv("E:\\Python\\pyspark_demo01\\out_dat\\aaa_1.csv", header=True, mode='append')

# 存为json
df_csv.write.json("E:\\Python\\pyspark_demo01\\out_dat\\aaa_1.json", mode='overwrite')

# 存为parquet格式--->占内存最少的
df_csv.write.parquet("E:\\Python\\pyspark_demo01\\out_dat\\aaa_1.parquet", mode="overwrite")

# 存为压缩文件
df_csv.write.csv("E:\\Python\\pyspark_demo01\\out_dat\\aaa_1.csv", header=True, compression='gzip', mode='overwrite')

spark.sql("show tables").show()
# 存成表格的形式
df_csv.write.saveAsTable("ratings_csv")
spark.sql("select * from ratings_csv").show()
